/*package com.wolfword.crawler.client.config;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileOutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;

import org.apache.http.HttpStatus;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.wolfword.common.enums.AnalyzedInfoTypeEnum;
import com.wolfword.common.exception.CrawlException;
import com.wolfword.common.util.object.StringUtil;
import com.wolfword.common.util.password.Base64Util;
import com.wolfword.common.util.password.GenerateShortUrl;
import com.wolfword.crawler.client.deal.StupidSpider;
import com.wolfword.crawler.client.dto.CrawlTaskWithHtmlDto;
import com.wolfword.crawler.client.entity.AnalyzedInfoListEntity;
import com.wolfword.crawler.client.entity.CrawlTaskEntity;

import edu.uci.ics.crawler4j.crawler.Page;
import edu.uci.ics.crawler4j.crawler.WebCrawler;
import edu.uci.ics.crawler4j.parser.HtmlParseData;
import edu.uci.ics.crawler4j.url.WebURL;

*//**
 * 处理html
 * 
 * @author renwei
 *
 *//*
public class CrawlerConfigBak extends WebCrawler {
	
	private final Logger logger = LoggerFactory.getLogger(CrawlerConfigBak.class);
	
	private String suffix;
	private String store_path;
	private String containStr;
	private long parentUrlId;
	private long rootUrlId;
	private String firstUrl;

	//UrlInfo ui = new UrlInfo();
	
	//private CrawlTaskEntity firstEntity = null;//处理的第一个任务，即相对第一级页面
	private StupidSpider singleSpider = StupidSpider.getInstance();

	private final Pattern FILTERS = Pattern
			.compile(".*\\.(bmp|gif|jpe?g|png|tiff?|pdf|ico|xaml|pict|rif|pptx?|ps"
					+ "|mid|mp2|mp3|mp4|wav|wma|au|aiff|flac|ogg|3gp|aac|amr|au|vox"
					+ "|avi|mov|mpe?g|ra?m|m4v|smil|wm?v|swf|aaf|asf|flv|mkv"
					+ "|zip|rar|gz|7z|aac|ace|alz|apk|arc|arj|dmg|jar|lzip|lha|js|css)"
					+ "(\\?.*)?$");
	
	@SuppressWarnings("unchecked")
	@Override
	public void onStart() {
		Map<String, Object> map = (Map<String, Object>) myController
				.getCustomData();
		containStr = (String) map.get("containStr");
		store_path = (String) map.get("storePath");
		suffix = (String) map.get("suffix");
		parentUrlId = Long.valueOf(String.valueOf(map.get("id")));
		rootUrlId = Long.valueOf(String.valueOf(map.get("rootUrlId")));
		firstUrl = (String) map.get("firstUrl");
	}

	@Override
	public boolean shouldVisit(Page referringPage, WebURL url) {
		String href = url.getURL().toLowerCase();
		return !FILTERS.matcher(href).matches() && href.contains(containStr);
	}

	@Override
	public void visit(Page page) {
		WebURL webUrl = page.getWebURL();
		String url = webUrl.getURL();
		if (null == url) {
			return;
		}
		// 获取短url
		String urlShort = GenerateShortUrl.shortUrl(url);
		String parentUrl = webUrl.getParentUrl();
		String parentShortUrl = null;
		
		if (!StringUtil.isEmpty(parentUrl)){
			parentShortUrl = GenerateShortUrl.shortUrl(parentUrl);
		}
		logger.error("URL: " + url);
		logger.error("short URL: " + urlShort);
		String fileName = "";
		try {
			if (StringUtil.isEmpty(store_path)){
				store_path = "./htmls";
			}
			// 用MD5值的第一个数做第一层目录，第二个数第二层目录，类推
			String storePathField = "/"+containStr.replace(".", "_")
					+"/"+ urlShort.substring(0, 1) 
					+ "/" + urlShort.substring(1, 2)
					+ "/" + urlShort.substring(2, 3);
			String path = store_path + storePathField;

			File file = new File(path);
			if (!file.exists()){
				file.mkdirs();
			}
			if (StringUtil.isEmpty(suffix)){
				suffix = ".html";
			}
			fileName = urlShort + suffix;
			String filePath = path + "/" + fileName;

			if (page.getParseData() instanceof HtmlParseData) {
				HtmlParseData htmlParseData = (HtmlParseData) page
						.getParseData();
				String htmlContent = htmlParseData.getHtml();
				//Set<WebURL> links = htmlParseData.getOutgoingUrls();
				
				
				// 将html内容写入文件
				// FileWriter bw=new FileWriter(file_path,true);
				// bw.write(html);
				Writer writer = new BufferedWriter(new OutputStreamWriter(
						new FileOutputStream(filePath), "GBK"));
				writer.write(htmlContent);
				writer.close();

				CrawlTaskEntity taskEntity = new CrawlTaskEntity();
				taskEntity.setUrl(url);
				taskEntity.setUrlShort(urlShort);
				taskEntity.setStorePath(storePathField);
				taskEntity.setParentUrlShort(parentShortUrl);
				taskEntity.setParentUrl(parentUrl);
				taskEntity.setRequestStatus(200);
				taskEntity.setContainStr(containStr);
				taskEntity.setRootUrlId(rootUrlId);
				
				if (url.equals(firstUrl)) {
					// 对第一级网页，进行数据分析
					List<AnalyzedInfoListEntity> entities = new ArrayList<AnalyzedInfoListEntity>();//分析出的信息列表
					Map<String, List<String>> analyze = analyze(htmlContent);
					
					List<String> list1 = analyze.get("mailCount");
					for(String str : list1){
						AnalyzedInfoListEntity entity = new AnalyzedInfoListEntity();
						entity.setType(AnalyzedInfoTypeEnum.MAILBOX.getValue());
						entity.setContent(str);
						entity.setTaskId(parentUrlId);
						entity.setUrlShort(urlShort);
						entities.add(entity);
					}
					List<String> list2 = analyze.get("phoneCount");
					for(String str : list2){
						AnalyzedInfoListEntity entity = new AnalyzedInfoListEntity();
						entity.setType(AnalyzedInfoTypeEnum.PHONE.getValue());
						entity.setContent(str);
						entity.setTaskId(parentUrlId);
						entity.setUrlShort(urlShort);
						entities.add(entity);
					}
					List<String> list3 = analyze.get("titleCount");
					for(String str : list3){
						AnalyzedInfoListEntity entity = new AnalyzedInfoListEntity();
						entity.setType(AnalyzedInfoTypeEnum.TITLE.getValue());
						entity.setContent(str);
						entity.setTaskId(parentUrlId);
						entity.setUrlShort(urlShort);
						entities.add(entity);
					}
					
					taskEntity.setPhoneCount(list1.size());
					taskEntity.setMailCount(list2.size());
					taskEntity.setTitleCount(list3.size());
					taskEntity.setId(parentUrlId);
					
					singleSpider.addCrawledResult(taskEntity);
					singleSpider.addAnalyzedInfo(entities);
				}else {
					taskEntity.setParentId(parentUrlId);
					CrawlTaskWithHtmlDto dto = new CrawlTaskWithHtmlDto(taskEntity);
					//dto.setHtmlStr(JSONSerielizer.toJson(htmlContent, null, null));
					dto.setHtmlStr(Base64Util.encrypt(htmlContent));
					singleSpider.addCrawledSubPages(dto);
					//singleSpider.addCrawledSubPage(taskEntity);
				}
				
				logger.info("visit:url--"+url);
				logger.info("visit:url_short--"+urlShort);
				logger.info("visit:status--200");
				
				//crawlerDao.addHtmlInfo(ui);
			}
		} catch (CrawlException e) {
			throw new CrawlException(e.getMessage(), e);
		} catch (Exception e) {
			System.out.println(e);
			e.printStackTrace();
			throw new CrawlException("插入爬取html信息到数据库异常");
		}
	}

	@Override
	protected void handlePageStatusCode(WebURL webUrl, int statusCode,
			String statusDescription) {

		String url = webUrl.getURL();
		// 获取短url
		String urlShort = GenerateShortUrl.shortUrl(url);
		String parentUrl = webUrl.getParentUrl();
		String parentShortUrl = null;
		
		if (!StringUtil.isEmpty(parentUrl)){
			parentShortUrl = GenerateShortUrl.shortUrl(parentUrl);
		}
		if (statusCode != HttpStatus.SC_OK) {
			CrawlTaskEntity taskEntity = new CrawlTaskEntity();
			taskEntity.setUrl(url);
			taskEntity.setUrlShort(urlShort);
			taskEntity.setStorePath("");
			taskEntity.setParentUrlShort(parentShortUrl);
			taskEntity.setParentUrl(parentUrl);
			taskEntity.setRequestStatus(HttpStatus.SC_NOT_FOUND);
			taskEntity.setContainStr(containStr);
			taskEntity.setRootUrlId(rootUrlId);
			
			if (url.equals(firstUrl)) {
				taskEntity.setId(parentUrlId);
				
				singleSpider.addCrawledResult(taskEntity);
			}else {
				taskEntity.setParentId(parentUrlId);
				CrawlTaskWithHtmlDto dto = new CrawlTaskWithHtmlDto(taskEntity);
				singleSpider.addCrawledSubPages(dto);
				//singleSpider.addCrawledSubPage(taskEntity);
			}
			
			logger.error("handlePageStatusCode:url--"+url);
			logger.error("handlePageStatusCode:url_short--"+urlShort);
			logger.error("handlePageStatusCode:status--"+HttpStatus.SC_NOT_FOUND);

//			if (statusCode == HttpStatus.SC_NOT_FOUND) {
//				ui.setStatus(404);
//				System.out.println("======================>");
//			} else {
//				ui.setStatus(403);
//			}
			logger.error("无法访问页面");
		}
	}
	
	*//**
	 * 对网页内容进行分析，获取网页包含的手机号、邮箱、职称等
	 * @param htmlContent
	 * @return
	 * @author ServerZhang
	 * @date 2018年1月15日
	 *//*
	private Map<String, List<String>> analyze(String htmlContent){
		Map<String, List<String>> map = new HashMap<String, List<String>>();
		
		List<String> phoneNos = StringUtil.getPhoneNos(htmlContent);
		List<String> emails = StringUtil.getEmails(htmlContent);
		List<String> titles = StringUtil.getTitles(htmlContent);
		
		map.put("phoneCount", phoneNos);
		map.put("mailCount", emails);
		map.put("titleCount", titles);
		return map;
	}

}
*/